{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "from torch.nn import NLLLoss\n",
    "import torch.optim as optim\n",
    "import torchvision.transforms as transforms\n",
    "import torchvision.datasets as datasets\n",
    "import torch.backends.cudnn as cudnn\n",
    "\n",
    "%matplotlib inline\n",
    "import pylab as pl\n",
    "from IPython import display\n",
    "import time\n",
    "\n",
    "from helpful_files.networks import PROTO, avgpool, covapool, pL, pCL, fsL, fsCL, fbpredict\n",
    "from helpful_files.testing import *"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Set Important Values\n",
    "\n",
    "# General settings\n",
    "datapath = './'                     # The location of your train, test, repr, and query folders. Make sure it ends in '/'!\n",
    "model = 'myModel.pth'               # What model do you wish to evaluate, and where is it saved?\n",
    "gpu = 0                             # What gpu do you wish to run on?\n",
    "workers = 1                         # Number of cpu worker processes to use for data loading\n",
    "verbosity = 10                      # How many categories in between status updates \n",
    "ensemble = 4                        # How many models to evaluate in parallel\n",
    "k = 1                               # Evaluate top-k accuracy. Typically 1 or 5. \n",
    "torch.cuda.set_device(gpu) \n",
    "cudnn.benchmark = True\n",
    "\n",
    "# Model characteristics\n",
    "covariance_pooling = True           # Did your model use covariance pooling?\n",
    "localizing = True                   # Did your model use localization?\n",
    "fewshot_local = True                # If you used localization: few-shot, or parametric? Few-shot if True, param if False\n",
    "network_width = 64                  # Number of channels at every layer of the network\n",
    "\n",
    "# Batch construction\n",
    "bsize = 64                          # Batch size\n",
    "boxes_available = 10                # Percentage of images with bounding boxes available (few-shot localization models only)\n",
    "include_masks = (localizing         # Include or ignore the bounding box annotations?\n",
    "                 and fewshot_local)\n",
    "n_trials = (10                      # Number of trials (few-shot localization models only)\n",
    "            if include_masks else 1)\n",
    "\n",
    "\n",
    "# Calculate embedding size based on model setup\n",
    "d = (network_width if not \n",
    "     covariance_pooling else\n",
    "     network_width**2)\n",
    "if localizing and not covariance_pooling:\n",
    "    d = network_width*2\n",
    "assert n_trials == 1 or include_masks, (\"Repeated trials will yield repeated identical results under this configuration.\"+\n",
    "                                        \"Please set ntrials to 1 or use a few-shot localizer.\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Data loaded!\n"
     ]
    }
   ],
   "source": [
    "# Load Testing Data\n",
    "\n",
    "d_boxes = torch.load('helpful_files/box_coords.pth')\n",
    "\n",
    "transform = transforms.Compose([\n",
    "    transforms.ToTensor(),\n",
    "    transforms.Normalize(mean=[0.4905, 0.4961, 0.4330],std=[0.1737, 0.1713, 0.1779])\n",
    "    ])\n",
    "\n",
    "repr_dataset = datasets.ImageFolder(\n",
    "    datapath+'repr', \n",
    "    loader = lambda x: load_transform(x, d_boxes, transform, include_masks))\n",
    "query_dataset = datasets.ImageFolder(\n",
    "    datapath+'query',\n",
    "    loader = lambda x: load_transform(x, d_boxes, transform, include_masks))\n",
    "repr_loader = torch.utils.data.DataLoader(\n",
    "    repr_dataset, \n",
    "    batch_sampler = OrderedSampler(repr_dataset, bsize),\n",
    "    num_workers = workers,\n",
    "    pin_memory = True)\n",
    "query_loader = torch.utils.data.DataLoader(\n",
    "    query_dataset,\n",
    "    batch_sampler = OrderedSampler(query_dataset, bsize),\n",
    "    num_workers = workers,\n",
    "    pin_memory = True)\n",
    "way = len(repr_dataset.classes)\n",
    "\n",
    "# Determine number of images with bounding boxes per-class\n",
    "catsizes = torch.LongTensor(np.array([t[1] for t in repr_dataset.imgs])).bincount().float()\n",
    "ngiv = (catsizes*boxes_available//100)\n",
    "for i in range(ngiv.size(0)):\n",
    "    if ngiv[i] == 0:\n",
    "        ngiv[i] = 1\n",
    "ngiv = ngiv.long().tolist()\n",
    "\n",
    "print('Data loaded!')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Ready to go!\n"
     ]
    }
   ],
   "source": [
    "# Make Models\n",
    "    \n",
    "models = [PROTO(network_width).cuda() for i in range(ensemble)]\n",
    "expander = avgpool()\n",
    "if localizing:\n",
    "    if fewshot_local:\n",
    "        expander = fsCL if covariance_pooling else fsL\n",
    "    else:\n",
    "        expander = pCL() if covariance_pooling else pL()\n",
    "elif covariance_pooling:\n",
    "    expander = covapool\n",
    "expanders = [expander for _ in range(ensemble)]\n",
    "\n",
    "# Load saved parameters\n",
    "model_state = torch.load(model)\n",
    "for i in range(ensemble):\n",
    "    models[i].load_state_dict(model_state[i])\n",
    "    models[i].eval()\n",
    "    # Zero out the bias on the final layer, since it doesn't do anything\n",
    "    models[i].process[-1].layers[-1].bias.data.zero_()\n",
    "\n",
    "# Load additional parameters for parametric localizer models\n",
    "if localizing and not fewshot_local:\n",
    "    fbcentroids = torch.load(model[:model.rfind('.')]+'_localizers'+model[model.rfind('.'):])\n",
    "    for i in range(ensemble):\n",
    "        expanders[i].centroids.data = fbcentroids[i]\n",
    "        expanders[i].cuda()\n",
    "\n",
    "print(\"Ready to go!\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#                                                    EVALUATE"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAABHEAAAAmCAYAAABDLAcdAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvOIA7rQAACUNJREFUeJzt3WusnVWdx/HvD9FysQakOKRaOJEqKgodjSCK4wUdDDoGjZdoEQhexguJF0QjEhWR8Y6oHQ0zjoqKIFFeGIzG+wwMQ7E1eCEZ5FoKeJAWS7kUhPL3xVpHNyfn2nJ62PD9JCftftazn+e/1t5v9i9rrSdVhSRJkiRJkh7YtpvvAiRJkiRJkjQ9QxxJkiRJkqQhYIgjSZIkSZI0BAxxJEmSJEmShoAhjiRJkiRJ0hAwxJEkSZIkSRoChjiSJGlaSb6e5GNb8f7bkjz+/qxJkiTpocYQR5KkbSjJL5P8OcmC+a5lrvQ+vmnwWFU9sqqumq+aJEmSHgwMcSRJ2kaSjADPBQp4+bwWo/tFku3nu4b7w4OlH5IkPdgZ4kiStO0cCVwEfB04arAhyY5JPptkTZJbklyQZMfednCSC5NsSLI2ydH9+H1mvCQ5OskFA68ryduTXJ7k1iQnJ9k7yf8l2ZjknCSPmOi9A+9fOr4TSXZNcl6Sm/qsovOSPK63nUILqlb0JVQrxl+rL8369yQ/6HWtTLL3wPX/OcllfRy+lOS/x8/sGTj3gN6fDUn+mGTFWJ96+75JfpLk5iQ3JjmhH39YkhOSXNlrWJ1kSZKRXuv2A9f42zj3cfrfJJ9LcjPwkT6mP0+yPsm6JGcm2WXg/UuSnNvHa32vcUGv6WkD5z0myaYku0/Qz6V9HG7p9/jODPq4IMlpSW7of6eNzQBL8vwk1yV5f5JR4Gv9+MuSXNLH88Ik+w3c5/1Jru/jdVmSQyb6TCRJ0twxxJEkads5Ejiz/x2a5B8G2j4DPAN4NvBo4H3AvUn2BH4IfBHYHVgGXDKLe76kX/dZ/Zr/ASwHlgBPBV63Bf3Yjvajfy9gT2ATsAKgqj4InA8c25dQHTvJNV4HnATsClwBnAKQZBHwXeADwG7AZbQxmcxm4N3AIuAg4BDg7f1aC4GfAj8CFgNLgZ/1972n13AY8CjgGOCOGfb/QOAq4DG97gAf7/d4Mm1sP9JreBhwHrAGGAEeC5xdVXcBZwNHjBuTn1bVTRPc82Tgx7Txehzt+zBdHz9I+9yXAfsDBwAnDlxzD9p3bS/gLUmeDnwV+Ffa2J8OfL+HQfsAxwLPrKqFwKHANb2Gg5NsmNnQSZKkrWGII0nSNpDkYNqP5XOqajVwJfD63rYdLUR4Z1VdX1Wbq+rC/kN/Oe2H/VlVdXdVra+q2YQ4n6yqjVV1KfB74MdVdVVV3UILh/5xtn3pNXyvqu6oqltpQcbzZnmZc6vq4qq6hxZqLevHDwMurapze9sXgNEpalldVRdV1T1VdQ0teBir5WXAaFV9tqrurKpbq2plb3sTcGJVXVbNb6pq/Qxrv6Gqvtjvuamqrqiqn1TVXT2AOXWghgNo4crxVXV7r2NsxtMZwOv75w/wBuCbk9zzbtr3Z/G4a0zVx+XAR6vqT72uk/o9xtwLfLjXvQl4M3B6Va3s38EzgLtoQdBmYAHwlCQPr6prqupKgKq6oKp2QZIkzTlDHEmSto2jaAHKuv762/x9SdUiYAdasDPekkmOz9SNA//fNMHrR872gkl2SnJ62tKvjcD/ALv0WSczNRjM3DFQx2Jg7VhDVRVw3RS1PLEv5xrttfwbbTxh6rHbmnFdO/iiL4M6uy812gh8a1wNa3ogdR89bLkdeF6SJ9Fm0Xx/knu+jzbj5+IklyY5Zgb9WEybATRmTT825qaqunPg9V7AcX0p1YY+u2YJLTi6AngXbYbRn3p/B68lSZK2AUMcSZLmWNreNq+h/Vgf7XuQvBvYP8n+wDrgTmDvCd6+dpLj0AKAnQZe77EVZd7nWkmmutZxwD7AgVX1KOCfxt7W/62tqOOPtOVCY3Vk8PUEvgz8P/CEXssJA3VMNXaTtd3e/51qXMf37+P92H69hiPG1bBnJt84+Ix+/huA744LVf5+w6rRqnpzVS2mLXf6UtoeQ1P18QZaMDNmz35ssn6sBU6pql0G/naqqrN6Dd+uqrEZZQV8cpL7SpKkOWKII0nS3DucthzlKbRlQ8toe6ecDxxZVffS9iI5NcnivunuQX0T2jOBFyV5TZLtk+yWZGzp0SXAK/vMmKXAG7eixt8A+yZZlmQH+p4uk1hIm8WzIcmjgQ+Pa78RePwW1vED4GlJDu/BxzuYOpxaCGwEbuuzWd420HYesEeSd/V9XRYmObC3fQU4OckT0uyXZLe+7Oh64Ij+ORzD5CHJYA230cbjscDxA20X04KpTyTZOckOSZ4z0P5N4BW0IOcbk90gyavTN48G/kwLUTZP08ezgBOT7N73GvoQbZbQZP4TeGuSA/uY7Jzkpf2a+yR5Yf9O3kn7/DdPMy6SJOl+ZogjSdLcOwr4WlVd22dUjFbVKG0z4OU9rHgv8DvgV8DNtFkO21XVtbR9Yo7rxy+hbVIL8DngL7TQ5Axa4LNFquoPwEdpm+ReDlwwxemnATvSZhBdRNtUd9DngVelPbnqC7OsYx3wauBTwHpa8LWKtjfLRN5L21voVloI8benNvX9el4M/Att+dblwAt686nAObTNgjcC/9X7BG1vmOP7/fcFLpym7JOApwO30EKocwdq2NzvvxS4lrY07LUD7dcBv6aFMudPcY9nAiuT3EZbcvXOqrp6mj5+jDZ2v6V9t37dj02oqlb1vq+gBUVXAEf35gXAJ2if+ShtU+exp2A9t9clSZLmWNpSc0mSpAeevunvdcDyqvrFfNczF5J8lbZZ8onTnixJkh7SJlufLUmSNC+SHAqspC3ZOZ62v8xF81rUHEkyArySLXhKmCRJeuhxOZUkSXqgOYj2xKV1tGVCh/dHYD+oJDmZ9tj3T1fV1fNdjyRJeuBzOZUkSZIkSdIQcCaOJEmSJEnSEJjVnjiLFi2qkZGROSpFkiRJkiTpoWf16tXrqmr36c6bVYgzMjLCqlWrtrwqSZIkSZIk3UeSNTM5z+VUkiRJkiRJQ8AQR5IkSZIkaQgY4kiSJEmSJA2BWT1iPMlNwIzWaUmSJEmSJGlG9prJxsazCnEkSZIkSZI0P1xOJUmSJEmSNAQMcSRJkiRJkoaAIY4kSZIkSdIQMMSRJEmSJEkaAoY4kiRJkiRJQ8AQR5IkSZIkaQgY4kiSJEmSJA0BQxxJkiRJkqQhYIgjSZIkSZI0BP4KVTMLqz9QxqoAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 1440x72 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracies and 95% confidence intervals\n",
      "Mean accuracy: \t\t34.20 \t+/- 0.20\n",
      "Per-class accuracy: \t30 \t+/- 0.19\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYgAAAEaCAYAAAAL7cBuAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvOIA7rQAAIABJREFUeJzt3Xe8XHWd//HXOwkhIQRIIBSBBAjhRoqEGmRR3B9KsYEICEhbhYi6iPJjBVRsuD+w7LJgQVCUslJEkCJYWLq6BEMREkpCDSWkEAhJgCDJ5/fH90zu5DL33pk77czM+/l4zGNmzpzyPWdmzud861FEYGZm1tOgZifAzMzyyQHCzMxKcoAwM7OSHCDMzKwkBwgzMyvJAcLMzEpygLCWIekiSd9pdjrMOoUDhJVN0tOSXpe0RNJcSb+UtGYN1y9JX5A0XdJSSc9JukrSdrXaRpnpeE+2j0uydETR+yWSxtZ4eztK+pOklyS9UeYygyQ9L+m+WqbFrJgDhFXqIxGxJrAjsAvwtUpXIGlILx+dA5wIfAEYDWwFXAt8aGBJHZiIuCsi1sz2c5ts8jqFaRExu8abXAZcDhxfwTIfANYEtmtCAO3t+7M24wBhAxIRzwO/B7YFkLS2pAslzcmubL8jaXD22TGS/iLpbEkLgW/2XJ+kCcDngcMi4taIWBYRr0XEryLirBLzj5L0O0nzJb2cvd6k6PNjJD0pabGkpyR9Mpu+paQ7JC2StEDSlQPZf0ljJd0kaaGkmZKOLvrsLEmXS7o62/7fJG3T27oiYkZE/BJ4pIIkHA38Brg5e12ctvUkXSLpxezYXFn02cGSHpT0qqRZkvbKpr8oaY8e+/Dz7PVESW9JOk7Ss8BNkoZk+zdX0iuSbpPUVbT8CEnnSno2O9Z3ZMvcIum4HumdKWnfCvbdGsQBwgZE0qbAB4H7s0kXA28BWwI7AHsDxxYtMhl4Elgf+PcSq9wLeC4i7ikzCYOAXwLjgLHA68CPsrSNAM4F9ouIkcDuwAPZcmcAfwJGAZsAPyxzez1dBTwGbAQcDpwt6Z+KPv846ZiMBq4DrikEzGpJWgs4APhV9vhkj3VfCQiYCGwA/Dhb7j3ABaRc2jqkY/5smZsdTPoOu4D9s2nXA+OBDYFHSftbcG62/V1Ix+BrQGTzHFG0L5OBtUiBzvImIvzwo6wH8DSwBHgFeAb4CTCcdBJaBgwvmvcw4Lbs9THA7H7W/VXg7n7muQj4Ti+fTQJezl6PyNL48eI0ZZ9dQjpJblLmPm9GOrENKZo2AXijx/6eDfw0e30WcHvRZ0OAl4Bd+tnWtsAbZaTpWOB5UpAcASwlBUOAzYE3gZEllrsYOLOXdb4I7FH0/izg59nridkxeEcfadoQWAEMA1YD/gF0lZhvBPAqMDZ7/yPgP5v92/aj9MM5CKvUARGxTkSMi4jPRcTrpKv41YA5WXHDK8D5pNxCwSpXqpJmFFX6vod0At2o3ERIWkPS+ZKekfQqcCewjqTBEbEU+ASpTH+OpBslTcwW/TLp6vqeLA2fGsAxeAcwP9v3gmeAjYver9zfiHgLeAF4h6RPF+33bwewbUhFSldExIpsX6+ju5hpU2BeRCwusdymwBMD3OaKiHih8CYrLvpBVoz3KikHIWBd0vc4hJRjXEWW3mtIuZ7VSN/TpQNMk9WZK5usFp4l5SDWy06GpawybHBErFImL+lF4MeSdo6IaWVs8/+SijsmR8SLkiaRiruUrf+PwB8lDQe+A/wMeE9EvAgcl21zD+B/JN0ZEY+Xua+QTvZjJA0vChJjSVf1BZsW7dtgUlB5ISKuAy6sYFurkDQe2INUOf3JbPIawGqS1iZ9F+tLWjMilvRY/FlSkVApS7P1FGxIKjIs6Dns87+QihH/GZhNykXOIR3/OdmyW5CK4Xq6mJRzmA7MjYj7S8xjOeAchFUtIuaQyvX/Q9JaWRPM8ZL2rGAds0hFVpdLep+koZKGSTpU0qklFhlJqnd4RdJo4BuFDyRtIOmjWV3EMlKx2PLss4OLKrNfJp34lle4y48DDwLfkbS6pB1JV/C/Kppnd0kfzq6Sv0zKIZVskqpkGDA0ez9M0tBetn0U8HdSsc+k7NGVrf+QiHiKlJv6kVLDgaGS3pst+3PgM5Lem31Hm0raKvvsAeCwLGewG931DL0ZSSpme4lUbLSyf0pE/INUlHdO9l0MlrRHUT3J7aQWWP+ezWc55QBhtXIU6QT3MOnE+xsqKDLKfIF0ZfljUh3CE8DHgBtKzPtfpPqPBcDdwB+KPhtEymG8ACwE9gQ+l322CzBV0hJSJeuJ2Um1bBERwCHA1qSy+yuBf4uIu4pmuxr4FOlYfBz4eET0Foi6SMHuXmD17PWDPWeSJNJx/nFEvFj0mEOqVykUMx1GKvKblaXvs1m67yIVu/0EWATcQqqoB/gKsB3puJ8GXNHPYbgQmJ+t/yHgzz0+/wLp+7ufFETOoDt3F6RipW2Ay/rZjjWR0ndlZrUi6SxScdux/c7coSRNIeV43t/stFjvnIMws4bKiv4+S8r1WI7VLUBI+oWkeZKmF00bLenmrIPOzZJGZdOVdap5XKkTz471SpeZNY+kjwLzSPU4v2lycqwfdStiyirGlgCXRESht+33gIURcVZW8TgqIk6R9EHgBFLHq8nAORExuS4JMzOzstQtBxERd5IqCIvtT3dvy4tJvUEL0y+J5G5Se/ZKKzjNzKyGGl0HsUHW4qLQNLLQkWpjVu1I9RyrdjoyM7MGy0tHOZWYVrLsK2v9MAVgxIgRO02cOLHUbGZm1ot77713QUSM6W++RgeIuZI2iog5WRHSvGz6cxT1PCW1zX7hbUsDEXEBWeuHnXfeOaZNK6fTrZmZFUh6ppz5Gl3EdD3dnXmOJo0hU5h+VNaaaTdgUaEoyszMmqNuOQhJlwPvA9aT9BxpKISzgF9L+jRp/JaDs9lvIrVgehx4jTTOi5mZNVHdAkREHNbLR3uVmDdIN4sxM7OccE9qMzMryQHCzMxKcoAwM7OSHCDMzKwkBwgzMyvJAcLMzEpygDAzs5IcIMzMrCQHCDMzK8kBwszMSnKAMDOzkhwgzMysJAcIMzMryQHCzMxKcoAwM7OSHCDMzKwkBwgzMyvJAcLMzEpygDAzs5IcIMzMrCQHCDMzK8kBwszMSnKAMDOzkhwgzMysJAcIMzMryQHCzMxKcoAwM7OSHCDMzKwkBwgzMyvJAcLMzEpygDAzs5IcIMzMrCQHCDMzK8kBwszMSnKA6MPp105n/Gk3cfq105udFDOzhmtKgJD0JUkzJE2XdLmkYZI2lzRV0ixJV0oa2oy0Fbts6myWR3DZ1NnNToqZWcM1PEBI2hj4ArBzRGwLDAYOBb4LnB0RE4CXgU83Om09HT55LIMlDp88ttlJMTNruCFN3O5wSf8A1gDmAP8HODz7/GLgm8B5TUld5owDtuWMA7ZtZhLMzJqm4TmIiHge+AEwmxQYFgH3Aq9ExFvZbM8BG5daXtIUSdMkTZs/f34jkmxm1pGaUcQ0Ctgf2Bx4BzAC2K/ErFFq+Yi4ICJ2joidx4wZU7+Empl1uGZUUr8feCoi5kfEP4BrgN2BdSQVirw2AV5oQtrMzCzTjAAxG9hN0hqSBOwFPAzcBhyUzXM0cF0T0mZmZplm1EFMBX4D3Ac8lKXhAuAU4CRJjwPrAhc2Om1mZtatKa2YIuIbwDd6TH4S2LUJyTEzsxLck9rMzEpygDAzs5IcIGxAPE6VWftzgOgQtT6he5wqs/bnANEhan1C9zhVZu2vWWMxWYMdPnksl02dXbMTusepMmt/iig5okVL2HnnnWPatGnNToaZWUuRdG9E7NzffC5iMjOzkhwgzMysJAcIMzMryQHCzMxKcoAwq4A7CFoncYAwq4A7CFoncYAwq4A7CFoncT8IM7MO434QZmZWlX4DhKSDJY3MXn9N0jWSdqx/0szMrJnKyUGcHhGLJe0B7ANcDJxX32SZmVmzlRMglmfPHwLOi4jrgKH1S5KZmeVBOQHieUnnA4cAN0lavczlzMyshZVzoj8E+COwb0S8AowG/q2uqTIzs6YrJ0CcHxHXRMQsgIiYAxxZ32SZmVmzlRMgtil+I2kwsFN9kmNmtebhQWygeg0Qkk6TtBh4l6RXs8diYB5wXcNSaGZV8fAgNlC9BoiIODMiRgLfj4i1ssfIiFg3Ik5rYBrNrAoeHsQGqqyhNiRtDIyj6B7WEXFnHdNVFg+1YWZWuXKH2hjS3wySzgIOBR6mu09EAE0PEGZmVj/9BgjgY0BXRCyrd2LMzCw/ymnF9CSwWr0TYmZm+VJODuI14AFJtwArcxER8YW6pcrMzJqunABxffYwW+n0a6dz2dTZHD55LGccsG2zk2MV8vdn5ei3iCkiLi71aETiLL/ctr61+fuzcvTVUe7X2fNDkh7s+WhcEi2P3La+tfn7s3L02g9C0kYRMUfSuFKfR8QzdU1ZGerdD8LZcDNrR1XfcjQblK8QCN4Atsser1cbHCStI+k3kh6V9Iikd0saLelmSbOy51HVbKMWnA03s05Wzi1HDwHuAQ4mDf09VdJBVW73HOAPETER2B54BDgVuCUiJgC3ZO+bytlws971HASwVQcFbNV0N0I5/SC+CuwSEUdHxFHArsDpA92gpLWA9wIXAkTEm9l9JvYn3c6U7PmAgW6jVs44YFueOPODLl5qkGr+qJ32J+9tfxt5HHrmsFs1x92q6W6EcgLEoIiYV/T+pTKX680WwHzgl5Lul/RzSSOADYqKteYA61exDWtB1fxRO+1P3tv+NvI49Mxht2qOu1XT3QjlnOj/IOmPko6RdAxwI3BTFdscAuxIur/1DsBSKihOkjRF0jRJ0+bPn19FMixvqvmjdtqfvLf9beRx6JnDbtUcd6umuxHKHc31QGAPQMCdEfHbAW9Q2hC4OyI2y96/hxQgtgTel7Wc2gi4PSK6+lpXO4zm6pZSZtZoVbdiylZygKSTgaURcVJEfKma4AAQES8Cz0oqnPz3Io0Uez1wdDbtaFrkpkTVlvl2WtGImbWOvjrK/QT4ErAucIakAVdMl3AC8Kusw90k4P8BZwEfkDQL+ED2PveqPcF3WtGImbWOvjrKTQe2j4jlktYA7oqIXN2LOg9FTC4iqk4jj99At+Xv2NpNLYqY3oyI5QAR8Rqp/qGjlFN85AquyvQ8po0sYhvotlwMaJ2qrwAxsWjspYeK3j/UKWMx+cRQe5fe/QzLI7j07tQZv5FFbAPdlosBLW8a1d+lryKmkmMwFXgsJhuIzU+9kSBlR58660PNTo5ZSxp/2k0sj2CwxBNnfrDi5au+J3UeAkCznXHAth0ZGOoZGI/YbdzKdZvZwBw+eWxD/kdl9YPIqzxUUrejaq9O2plzldYOatIPwjqTy9x753op6yRlBQhJw4s6tlmRdhwkzi2zeufgaZ2k3yImSR8BfgAMjYjNJU0Cvh0RH21EAvuShyImF8eYWaupZRHTN0lDfL8CEBEPAJtVk7h24itKM+tNq5cwlBMg3oqIRXVPSQPV8ktzcYw1U6ufgNpdq9dZlRMgpks6HBgsaYKkHwJ/rXO66qrcL81/vnzx9/F2rX4CanetXsJQToA4AdgGWAZcDrwKfLGeiaq38euPWOW5N/7z5ctAvo92DyqtfgJqd61ewtBvgIiI1yLiqxGxS0TsnL1+oxGJq5cn5i1d5bk3/vPly0C+j3YP8q1+ArJ867UndYGkG4CeTZ0WAdOA81sxWJTbC7FTe1Ln1UC+j0b1ODVrR+U0cz0HGEMqXgL4BPAiMBxYKyKOrGsK+5CHZq5mncg9yltbLZu57hARh0fEDdnjCGDXiPg86d7SliPtXubeSD6WvWv3ojtLygkQYyStzJ9nr9fL3r5Zl1TZgPmPWzs+lm9XCJrj1x8x4Po5B97WUU6A+L/AnyXdJul24C7g3ySNAC6uZ+Kscq5Yr52+jmWnnuQKQfOJeUsHXDler8Dbqd9JPZU1mquk1YGJpGH8H81LxbTrIFpPu5Rd12OIlVY4NrVIY73208PelK/Wo7lOALqAdwGHSDqqmsRZ52qXYpt65NRa4djUolltvZrmOvdce/0GCEnfAH6YPf4Z+B7Q9IH6rDU14k/ciKKGepzkfIKrTjnfiYuhKlNOM9eHgO2B+yNie0kbAD+PiI80IoF9cRGTldJqRQ2tULTULlrtt1EvtSxiej0iVgBvSVoLmAdsUW0Czeql1a7EW6FoqV30/G04R9G3cnIQPwG+AhxKatG0BHggIv6l/snrm3MQ1g6cg2ieTs1RlJuDqOie1JI2I/WefnDgSasdBwjrj0++1pdO/X3UrIhJ0i2F1xHxdEQ8WDzNrDeVZt/rkd138U3vXLziwQ7702uAkDRM0mhgPUmjJI3OHpsB72hUAq11VXpyrsfJvNXqIxrJwdP601cO4jPAvaQOcvcWPa4Dflz/pFmrq/Tk7JN5Y/l4W3/KqaQ+ISJ+2KD0VMR1ENafTq2E7E2nlrnbqsqtg+j3fhAR8UNJuwObFc8fEZdUlUKzBvD9IFZVXKzkANFCliyBmTPhscfS45//Gfbcs+6bLeeGQZcC44EHgOXZ5AAcIFpcb1eT7XSV6Zs+rcoBszEG9B9avhxmz04BYPRo2HVXWLQIttkGnn++ez4Jhg1rSIAop4jpEWDrqKQ9bIO0QxFTM0/GvRW/bHbqjStfP33WhxqaJrN20GfR5ssvpxP/Zpul90ceCQ88ALNmwbJladoRR8Cll0IEHH88jBsHXV3pseWWKUBUoWZFTMB0YENgTlUpspKameXv7WpSpCyiGpoas/bxyV024Vd/ey79t37xC/jrX7uLh+bPT1f/t9+eZl60CDbfHPbdtzsITJyYPpPg/PObth/lBIj1gIcl3QMsK0yMCA/YVwPNzPL3VvxyxG7japKmdiqqMuvVgw/CPfd0B4DHHuPbgwbx7UceSZ/v929w773pxP/Rj6bnSZO6l7/++uakuwzlFDGVLOiKiDvqkqIKtEMRU19a/QTrFkT11+q/kZZI/xtvwOOPrxIAeOqplAMYNAimTIGf/QyGDoUJE1IA2GYb+Na3Ug5g2TJYffWqklDr41SzntRZIHgaWC17/TfgvhokcLCk+yX9Lnu/uaSpkmZJulLS0Gq3kQfV9FZt9Y5MbmffrV69llv9N5Kb9EekiuBbb4XzzoMvfhEWLkyffe97sN12cNBB8NWvwi23wGqrwauvps+/8hV44gl47TWYPh2uvhq+/e0UHKDq4ADNO07lDLVxHPAboFAQtjFwbQ22fSLwSNH77wJnR8QE4GXg0zXYRtNV88W2+gnWwxh0q9cfvNV/Iw1P/5IlcN99cPnl3S2Drr0W1loLNtkE9toLPve5lCN4+un0+YEHwmWXpWKixYvhuedSkFhnnfT5ZpvBFlvA4MF1S3azvudyipgeAHYFpkbEDtm0hyJiuwFvVNqEdD/rfwdOAj4CzAc2jIi3JL0b+GZE7NPXelqhiKleWeiWyJrbSv6+GmjFitRcdMQIGDMGHn0UPv/5VDRU3Fz08svh0ENhxgy44ILuCuKuLth44+4cQBuq2WiukqZGxGRJ90fEDpKGAPdFxLuqSNxvgDOBkcDJwDHA3RGxZfb5psDvI+Jt/yRJU4ApAGPHjt3pmWeeGWgyWprL91ufg0aVli9PV+2LF8NZZ3XXDxSai37/+3DyyfDss6l4qDgAdHWl+oIqm4u2qlo2c71D0leA4ZI+AHwOuKGKhH0YmBcR90p6X2FyiVlLRq6IuAC4AFIOYqDpaHXu8FS5vJ2Q3au5TBFw440pJ1BcUXzooXDOOamM/z/+A8aOTSf+ffZJz+99b1p+001h6tTm7kOLKidAnEqqD3iINIDfTcDPq9jmPwEflfRBYBiwFvBfwDqShkTEW8AmwAtVbKPtuYdw5ep9Qq40ANUzyOctGPZrwYK3B4DNN4f/+q9U1DNlCsyZk4qMurrgIx/pDgBDh6a6hSHlnM6sEuUUMY0A3oiI5dn7wcDqEfFa1RtPOYiTI+LDkq4Cro6IKyT9FHgwIn7S1/KtUAdh+VHvk2Yji/3625dcFkH2aC5660PPc9xmH2L8+iP4/vensP2cmWm+QnPRvfeG//zPNO3hh2GjjWDUqOalv43U8p7UtwDDi94PB/5noAnrwynASZIeB9YFLqzDNtqebwLTu3q3qmpkS5NCbujSu58p+X03rXVTobnobbfBRRd1T//sZ1OlcVFz0U3+53csj2Dm3CX85x6f5FMHf2vV5qKF4ACw9db9Bgf/9muvrFZMETGpv2nN4BzE2+XyytFqrpCDWBFBQOO/76VL0+ii226b+gRcckmqD5g5MxX3FCxalJqQXnEFPPJIdwXxVltx+i3PcNnU2YxffwRPzFtadc7Ov/3y1bKSeqmkHSPivmzFOwGvV5vAZmq58tkKuPK6MxTqoIp/yzW3YkXKEQwenPoOXHhhd/3Ac8+leR5+GN75zlT+P2YM7LEHbLVVdyAYOTLNd+ihve5Drfi3X3vl5CB2Bq6ku9J4I+ATEXFvndPWr4HmIFrlSqOdA1kpnba/1aj5sVqwAH7/+1UriWfNSuMEfeAD6fmoo1bJAdDVlT4rdBizllGTHISkQcBQ0m1Hu0jNUR+NiH/UJJVN0ipXGu3YDLKvE1s77m+9VHysli9P5fvFAeCxx+Bf/xUOOST1FTjqqJRb2GKLdPLfe294R3b7+Q9/OA1T3cadx+zt+gwQEbFC0n9ExLtJw363hXKzts2+om2VQFaJvk5s7bi/9dLrsVqwYNUAsNNOKQAsXJhO+gXrrZfeD8raqWyzTaoj2GKL1Iqop0HltGexdlNOEdO3gAeBa/J206B6V1K3SlFUI9QqWDY76LaFZcu6m4sOHw777ZfqCsaO7a4bgHSiP/HENNhcBPz3f6ebzXR1pTuWWceq5VAbi4ERpNuNvk52P5mIWKsWCa1GvQOET2bdHCwbLCJ1DFuwAN6VjWozZUoaJO7pp1MFMqTOYndkI+9/+9upxVChnmDcuJoPIOf/RO0081jWLEDkmZu5Nk4jfswdefJZtozTfz+Ly6bO5ruv/52D5k/vLh5asiT1Jn7yyTTviSfC3LmrVhRvtVUKCg3iC4XaaeaxrFkzV0kCPglsHhFnZAPpbRQR99QgndYiGjG0RyUVr7UMJg0LTA8+mG4yM3NmdxBYsIArTriS5RrEsjv/DAtnpJP/v/xLen7nO7uXP+ecqpNQ7b66nqh2WuFYllPEdB6wAvg/EfFOSaOAP0XELo1IYF+cg8iXak8+lSxfy6uv3tZV8f4sWZIqenu2FLr5Zlh/fTjjDPj611ctBurq4lsT9uWSB+Zx+K6bcsbHBjyKflmcA2gN9b5oqWVHuckRsaOk+wEi4uV2udtb3tX7R1Lr9Q+kmWrPNJS73Pj1RzBz7hLGrz+imiQDvV/Jldyft95Kt5ssDgAnnZRuMn/VVfCpT6X5ipuLLl2apn32s3DccbDBBqs0F/0G8I1PVL0bZWmFq1bLT5Pvctqu/SMboC8AJI0h5Siszup9m8Far38g4/8MNA0z5y5Z5bkaJcdoWrCAU9d6iU88eDNf3OjNNO0vf4E11kjl/h/5CJx8MkuvuAqye5J8f/mmfObAr3HOD69L4wnNnAk33JDqESA1Ld1ww6b2JehtPCqPY5QveblTYDk5iHOB3wLrS/p34CDga3VNVQup51V+va/2ar3+gdRTDDQNIl2xVHWqXbYsdR4bPjydxOfOhQMOSCf2hQs5rjDf+7LOYltumXILXV0ceMtLPLnuxrwyfC2e3ifd+PCnT7zJ8gm78T/PixNL9SXIsbxcsZar3Rs05GU4/7JaMUmaCOxF+j/eEhGP9LNIQ+ShDsJlus1R9gkiIl3NjxiRmoaefHL3fQcKzUVPOAHOPRfefBP23XfVsYQKzUV73Gtg81NvXBmgnjrrQ5WlKYdaLe3+31Wn6maukoYBxwNbkm4WdGF2M5/cyEOAaLU/Vtu7/np44IHu+oGZM9ON6H/72/T5VlulYqLiALDzzqv2Mi7D3mffwcy5S9hqgzX505f2rMOOWF96/u/8P6xMLQLElcA/gLuA/YCnI+KLNU1llfIQIKzBnn8eZszgd1fdxsL7prP78gVsOW59uO669PnkyXDPPd23n+zqgt13h8MOS59HlF0H0NdJZ7NTb1z5+uksB2EDU4uTu3MUlalFK6atI2K7bGUXAu73YI2xaNGqrYRefBF+nt3l9sQT4eqr+TCweOhwnhq9Cby76NYkV18N666b6hVKqaCCuPimPD1PYDWpAzGgNvUfbp1VH321Ylo5Ymveipas8a1Oar69t95K4wndeGO6c9iyZWn6V7+aho+ePDmNLnrmmWkoidezW5CccgrcdhvfvfBWJn3pKq467xo477zu9W6ySe/BocL9KbQkEbytpdURu41jsMQRu40byN5bkVq02Kn13QLdqivpq4hpObC08JZ0q9HX6KCxmPKs0VnqAW+vMLrottvC2mvDtdfCV76SgsM/ikaNnzGD02eu4Mlf38DRQ+ax94F7puKh8eNLjy7awP1x+Xbnafciq6rvSR0RgyNirewxMiKGFL1uenDodI1uJ93n9pYt677Cf/TRNEzE7runop7CXcbuvjt9PmpUOvGfdBL84hepb8GCBbD11lw2dTZ/GfsuPrvxB1Jz03e+sy7Bod/96aHe97K2/Onv99EpOQwP1meVefVVuPzyVesInnoKzj8fjj0W/v73NPx08V3Hurrg3e/ud4hpX6nbQDTjd9PqOYxaDrVhnWTFCnjoobeNJ/SHrd/L59+xF8e+cySnHX98KuefMAF23DG1ENpxx7T89tvDCy/0vQ2zGmpGJ79OqRR3gOhEK1akG8sUB4Hx4+GLqRXzm7vuxtA330jzZs1Fb12wguUbBT9/ZDGnPfNMqgyu8V3GWq03b704J1WZZpys89LTud4cINrZq692B4Dly+Hoo9P0SZNSLqFg5Ej4RDZa3KBBfOaA03hxxChmj96YGT84EIDVr53O4MKfcGx9/og9B+Dr1BNlXgNlXr+PTjlZN4NvNJtjZVWEFZr1esYGAAAPV0lEQVSL3n77ymUu3PVjLB49JrUa2nVXOPLIdLexgi98AX76U7jttlQctGgR/OxnKz/e5LADmbnBeA7cY6uV0xpRUfvEvKWrPNd7sMK8qlUDhGorUnsu36nfRydzgMixVf6QCxemXsAAv/41fOxjsPXWadiICRNgn31g+XIumzqbF0eM5o+bTEp9CK65BmbMgIcf7l7xscfCZz4D73sfbLTR2zqPNavVTs8TY15GtGy0Wh3/ak/oPZcv5/volNY9ncKtmPJoxgz43e+49093w2OPMfHVOYxY/Eq6R/GGG6ab0P/yl6uOJ9TVBbvtxuk3PJLLYoB6y2vxRzM18gZOBa3euqdT+J7UebZ4Mdx339vvPHbFFbDTTnDxxXDMMSkYFAeAY47pt6lop/KJKR9aOVC3ctor5WaufWjID+G112DWrFUDwHHHwZ57psHk3v/+NN+wYam/wA47dA8pfdBBqaPY2mvXJ21tqJ4tWar5vXTSSQdau8I4r40DmqkjcxA1u9rs2Vx0u+1SAHjiiXRzmWKbbpqKhg49NFUKT52acgWbblrz5qJWW9X8XuqVs+m0wNMInXRMqx5qo51VXPm5eDFMm5buMwBpDKEddoA110w3k9l773TTmWuvTZ+PHQvf+lYqMrr//nQz+9mzU3CAlDPYe++07ACDQz0rA0utu5MrH6upLK9XRbtbFNWeh1R5u47MQZS0fHlqKTRmTHp/yimpKOixx1LlMMD++3cHgSOPTPcY7upKN6zv6ip5v+F6XZVUemVaSTpKrXsgV8KddEVWD30dPx9bq4ZzEP254QY47TQ48EDYZpvUXHS//bo/nz49DUK3zz7dzUV/8IPuzy+9FM4+G44/vtfmotD/ld5Ar8wrvTKt5Iqz1LoHciXcrKvcdsnt9HX8fLVrjdC5OYhDDkm5gfHju1sJbb89HH54TdPY35Veo1rfNOOKs1lXuXls0TSQY9HpuYS87X/e0lMNN3Ptz8svpyEmhjS3IVc7/ejyolbHtJbfTR6DVt7l7ZjlLT3VyG0Rk6RNJd0m6RFJMySdmE0fLelmSbOy51F1TcioUTUJDtUWZ/RVVJDnopI8p63WPZEvvfuZqve1U3uFVyNvxyxv6WmEhucgJG0EbBQR90kaCdwLHAAcAyyMiLMknQqMiohT+lpXHjrK1fOqIs9XLHlOW60UchArIgho6321zpLbHEREzImI+7LXi4FHgI2B/YGLs9kuJgWN3KvnVUWer1jyPC5PNdstXraQEyncfzqP34NZPTW1DkLSZsCdwLbA7IhYp+izlyOiz2KmPOQgqtXOdRDl5DLqsf957NhWL+38+2kHef1+cpuDKJC0JnA18MWIeLWC5aZImiZp2vz58we07TyVn7dzh6dychn12P88dmwrpRa/w3b+/bSDVv9+mhIgJK1GCg6/iohrsslzs/qJQj3FvFLLRsQFEbFzROw8ptCprUJ5+tLyWIxUqwBaTmVxPfa/mkrqRvYvqMXvMI+/H+vW6t9PMyqpRapjWBgRXyya/n3gpaJK6tER8eW+1jXQIqa8Zvsarbfj0GrFLPVS79+Jf4fWLLntByFpD+Au4CFgRTb5K8BU4NfAWGA2cHBELOxrXfWug2j3P3BvgaDd97tcDpTWrnJbBxERf44IRcS7ImJS9rgpIl6KiL0iYkL23GdwqEa5RSh5Koqqh96yvx7GIWn14gGzanVkT+pyrwx9JW1m7cg3DOpDuTeXaeWbn5iZVasjcxCN5FxIY/g4m5Uvt3UQnabd6zGKNbN/ST2Oc576y5g1gwNEH2pxgmh0RWe7naTLVY/j3EnB3awUFzH1oRWbOTbz/hLtUMxTvA9Ay++PWSkuYqqBVmzm2Kg0l7q6bofmscX7VelQ7JXm3lyEZXnnANGHck54efuTV3KSribt9QhEeTiW5e5XqQBZaZGUi7As7xwgqlSve043Qm9pLyfN9cgt5OGEWe5+1eK+3a2YQ7XO4gBRpf7+5HkOIL2lvVkn6lbKlZQKJJUGzXYokrP25krqOuuv4naglcr1rBBuh8rmglZsaNBOx9/yyZXUOdHfVeJAr5qrKR6qNs2tpBWLcfJQ1GYGHRog8lQvMNCTcd6Kh/KqFYNdKwY1a08dWcTUisUO5XLxhJn1x0VMfWjnK7RmXjHnKWdWL52wj2YFHZmDsPpo55xZQSfso7U/5yBqoJyrRV9RdmvnnFlBJ+yjWYFzEH0o52rRV5SN04j6lTzU4eQhDdbenIOogfHrj1jluRRfUTZOI1po5aEVWB7SYAYOEH16Yt7SVZ5LacVmlK2qEcE4DwE/D2kwAxcx9Wnvs+9g5twlbLXBmvzpS3sOaB0uLmg8H3OzvrmIqQbKyUH0V0nt4oLGq+cxd6ME6yQOEH0oJ6vf38nIxQWNV89j7oBvnaQji5hqWQTh4ozO4u/b2kG5RUwdGSDcNNXMOpnrIPpQTvNVaM3y5lZMs5nlU0cGiHIqn6E1y5sblWYHIrP215EBotxKzDWHDV7luZR6nigHsu5GVYq3YvCslIOgdbqODBDldm5b9PpbqzyXcundz7A8gkvvfqamaYSBnYR727dSJ7tqToCd0DqrE4KgWV86MkDU8spQPZ5rqZYn4VInu2pOgPXoQZ63K/ZOCIJmfenIAFHuifHI3cYxWOLI3cb1Os8R2TxH9DHPQPV1Eq70ZFrqZJe3E2Dertg9jIp1uo4MELU8MTbrJFLpybRUOuud9loEMTNrno7sB1GuzU+9kSAVHz111ofqtp2BaIUOW+5vYpZP5faDGNKIxLSq6PGcJ2ccsG1uA0PB4ZPHrgxiZtZ6HCD6cORu43yCq0IrBDEz612u6iAk7SvpMUmPSzq12ekpp4y+WS1v8tbipxE6cZ/Nmik3AULSYODHwH7A1sBhkrZubqr616yWN5Vutx1Ornlr5WTW7nJTSS3p3cA3I2Kf7P1pABFxZm/LjBw5MnbaaaeKt/XUgqXMe3UZ66+1Opuv1/d4TI1cVz23O/XJhQSBEJO3GN2AFNZes461Wbu54447Wms0V0kHAftGxLHZ+yOByRHxrz3mmwJMyd52AY8Vfbw2sKi/bQ3dYMudlr++iMHD1+bNuY/fW5MdyLnBa28wdvCwkWOWv7F4/vJFc8u5BC/rWDZZs9JYz+3Wct3VrGugy1a6XCXzrwcsqDhFnaHS4z4uIsb0N1OeKqlLdUZ+W/SKiAuAC0quQLogIqaU+qzEvNPeWjSv3wjaqSo5ls3SrDTWc7u1XHc16xrospUuV+l/tpyr3k5Ur99kbuoggOeATYvebwK8UOE6bqhdcjpeKxzLZqWxntut5bqrWddAl610uVb4nbWCuhzHPBUxDQFmAnsBzwN/Aw6PiBl12p6vRsxaiP+zjZebIqaIeEvSvwJ/BAYDv6hXcMiULKYys9zyf7bBcpODMDOzfMlTHYSZmeWIA4SZmZXkAGFmZiU5QGQkbSHpQkm/aXZazKx/kg6Q9DNJ10nau9npaUdtHSAk/ULSPEnTe0x/26CAEfFkRHy6OSk1M6j4P3ttRBwHHAN8ognJbXttHSCAi4B9iye06qCAZh3iIir/z34t+9xqrK0DRETcCSzsMXlX4PEsx/AmcAWwf8MTZ2ZvU8l/Vsl3gd9HxH2NTmsnaOsA0YuNgWeL3j8HbCxpXUk/BXYojCRrZrlQ8j8LnAC8HzhI0vHNSFi7y01P6gYqOShgRLwE+Edmlj+9/WfPBc5tdGI6SSfmIGoxKKCZNY7/s03SiQHib8AESZtLGgocClzf5DSZWe/8n22Stg4Qki4H/hfokvScpE9HxFtAYVDAR4Bf13lQQDMrk/+z+eLB+szMrKS2zkGYmdnAOUCYmVlJDhBmZlaSA4SZmZXkAGFmZiU5QJiZWUkOEJZLks6U9L5szP9Te5nnm5Kel/SApIclHVbGesdImirpfknvqX3KqyfpGEk/anY6zBwgLK8mA1OBPYG7+pjv7IiYRBqR93xJq/Wz3r2ARyNih4joa70rZcNNm3UcBwjLFUnfl/QgsAupR+2xwHmSvt7XchExC3gNGJWtZ7ykP0i6V9JdkiZKmgR8D/hglusYLmlvSf8r6T5JV0laM1v+aUlfl/Rn4OBS68vmu0jSuZL+KulJSQcV7cuXJT0k6e+SzuotXf0cj4sknSfptmz9e2Y31XlE0kVF850naZqkGZK+VTT9g5IelfTnLJ2/y6aPyNbztyw3tX82fRtJ92TH50FJE8r75qwtRYQffuTqQRr//4fAasBf+pjvm8DJ2esdgbuKPrsFmJC9ngzcmr0+BvhR9no94E5gRPb+FODr2eungS+Xsb6LgKtIF1tbk+5bAOnmNn8F1sjej+5rPT32qziNF5HufyBSLulVYLtse/cCk3qsfzBwO/AuYBhpmOzNs88uB36Xvf5/wBHZ63WAmcCI7Lh/Mps+FBje7N+DH817dOJw35Z/OwAPABOBh/uZ90uSjgO2ILsTWZYL2B24Slo5UvTqJZbdjXRS/0s231BSrqXgyjLXd21ErAAelrRBNu39wC8j4jWAiFhYQbp6uiEiQtJDwNyIeChL1wxgM9KxOkTSFNIQ/htl+zUIeDIinsrWczkwJXu9N/BRSSdn74cBY7P9/6qkTYBrIuXMrEM5QFhuZEVAF5GGc14ArJEm6wHg3RHxeonFzo6IH0g6ELhE0njSifGVSHUTfW4SuDkieqvcXpo997e+ZT3WWXjuOdBZuenqbf0remxrBTBE0ubAycAuEfFyVvQ0jNL3UShO58cj4rEe0x+RNBX4EPBHScdGxK0VptfahOsgLDci4oHs5DmTdAV8K7BPREzqJTgUL3sNMA04OiJeBZ6SdDCkCCNp+xKL3Q38k6Qts/nWkLRViXWXu75ifwI+JWmNbJnRA1xPOdYiBbNFWQ5mv2z6o8AWkjbL3n+iaJk/Aicoy8pI2iF73oKU6ziXNKT2u2qQPmtRDhCWK5LGAC9nRTYTI6K/IqZi3wZOkjQI+CTwaUl/B2ZQ4r7jETGfVN5/eVYxfjepWKuUftfXY91/IJ1gp2U5oEJRTkXrKUdE/B24P1vfL4C/ZNNfBz4H/CGrbJ8LLMoWO4NUx/OgpOnZe0hBZHqW5onAJdWmz1qXh/s2a2OS1oyIJVlO4cfArIg4u9npstbgHIRZezsuyw3MANYGzm9yeqyFOAdhZmYlOQdhZmYlOUCYmVlJDhBmZlaSA4SZmZXkAGFmZiU5QJiZWUn/H9k2xli3zWEaAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "acclist = []\n",
    "pcacclist = []\n",
    "alldispacc = np.zeros(way)\n",
    "for r in range(n_trials):\n",
    "    # Accumulate foreground/background prototypes, if using\n",
    "    fbcentroids = (accumulateFB(models, repr_loader, way, network_width, ngiv, bsize)\n",
    "                   if include_masks else \n",
    "                   [None]*ensemble)\n",
    "    # Accumulate category prototypes\n",
    "    centroids, counts = accumulate(models, repr_loader, expanders, \n",
    "                                   fbcentroids, way, d)\n",
    "    # Score the models\n",
    "    allacc, dispacc, perclassacc = score(k, centroids, fbcentroids, models, \n",
    "                                         query_loader, expanders, way)\n",
    "    # Record statistics\n",
    "    acclist = acclist+allacc\n",
    "    pcacclist = pcacclist+list(perclassacc)\n",
    "    alldispacc += dispacc\n",
    "\n",
    "# Aggregate collected statistics\n",
    "accs = sum(acclist)/n_trials/ensemble\n",
    "pcaccs = sum(pcacclist)/n_trials/ensemble\n",
    "alldispacc = alldispacc/n_trials\n",
    "confs = 1.96*np.sqrt(np.var(acclist)/n_trials/ensemble)\n",
    "pcconfs = 1.96*np.sqrt(np.var(pcacclist)/n_trials/ensemble)\n",
    "\n",
    "# Report\n",
    "print(\"Accuracies and 95% confidence intervals\")\n",
    "print(\"Mean accuracy: \\t\\t%.2f \\t+/- %.2f\" % (accs*100, confs*100))\n",
    "print(\"Per-class accuracy: \\t%.f \\t+/- %.2f\" % (pcaccs*100, pcconfs*100))\n",
    "logcounts = [np.log10(c) for c in counts]\n",
    "pl.figure()\n",
    "pl.axhline(0,color='k')\n",
    "pl.scatter(counts, dispacc*100, s=4)\n",
    "z = np.polyfit(logcounts, np.array(dispacc)*100, 1)\n",
    "p = np.poly1d(z)\n",
    "pl.plot([min(counts),max(counts)], [p(min(logcounts)),p(max(logcounts))], \"r--\")\n",
    "pl.ylim([0,100])\n",
    "pl.xlabel('# Reference Images')\n",
    "pl.ylabel('Percentage Points')\n",
    "pl.xscale('log')\n",
    "pl.title('Per-Class Top-%d Accuracy' % k)\n",
    "pl.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Shut down the notebook"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/javascript": [
       "Jupyter.notebook.session.delete();\n"
      ],
      "text/plain": [
       "<IPython.core.display.Javascript object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "%%javascript\n",
    "Jupyter.notebook.session.delete();"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
